# -*- coding=utf-8 -*-
# @Time : 2019/12/12
# @Author : rubick

import requests
from lxml import etree
import json

# http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/11/01/110101.html
start_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/'
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15',
}


def getProvince():
    response = requests.get(start_url + 'index.html', headers=headers)
    response.encoding = 'gb2312'
    html = response.text
    dom = etree.HTML(html)
    province = dom.xpath('//table[@class="provincetable"]//tr[@class="provincetr"]/td/a/text()')
    suffix_urls1 = dom.xpath('//table[@class="provincetable"]//tr[@class="provincetr"]/td/a/@href')
    return (province, suffix_urls1, start_url + 'index.html')

def getCityData(url):
    response = requests.get(url, headers=headers)
    response.encoding = 'gb2312'
    html = response.text
    dom = etree.HTML(html)
    data = dom.xpath('//table[@class="citytable"]//tr[@class="citytr"]/td[2]/a/text()')
    suffix_urls = dom.xpath('//table[@class="citytable"]//tr[@class="citytr"]/td[2]/a/@href')
    return (data, suffix_urls, url)


def getCountryData(suffix):
    response = requests.get(suffix, headers=headers)
    response.encoding = 'gb2312'
    html = response.text
    dom = etree.HTML(html)
    data = dom.xpath('//table[@class="countytable"]//tr[@class="countytr"]/td[2]/a/text()')
    suffix_urls = dom.xpath('//table[@class="countytable"]//tr[@class="countytr"]/td[2]/a/@href')
    return (data, suffix_urls, suffix)

def getTownData(suffix):
    response = requests.get(suffix, headers=headers)
    response.encoding = 'gb2312'
    html = response.text
    dom = etree.HTML(html)
    data = dom.xpath('//table[@class="towntable"]//tr[@class="towntr"]/td[2]/a/text()')
    suffix_urls = dom.xpath('//table[@class="towntable"]//tr[@class="towntr"]/td[2]/a/@href')
    return (data, suffix_urls, suffix)


def getVillageData(suffix):
    response = requests.get(suffix, headers=headers)
    response.encoding = 'gb2312'
    html = response.text
    dom = etree.HTML(html)
    data = dom.xpath('//table[@class="villagetable"]//tr[@class="villagetr"]/td[3]')
    return data

address = []
province, url, last_url0 = getProvince()
for i in range(len(province)):
    add0 = province[i]
    suffix0 = url[i]
    next0 = []
    suffix0 = last_url0.replace(last_url0.split('/')[-1], '')+suffix0
    print(suffix0)
    data1, url1, last_url1 = getCityData(suffix0)
    for j in range(len(data1)):
        add1 = data1[j]
        suffix1 = url1[j]
        next1 = []
        suffix1 = last_url1.replace(last_url1.split('/')[-1], '')+suffix1
        print(suffix1)
        data2, url2, last_url2 = getCountryData(suffix1)
        for k in range(len(data2)):
            add2 = data2[k]
            suffix2 = url2[k]
            next2 = []
            suffix2 = last_url2.replace(last_url2.split('/')[-1], '') + suffix2
            print(suffix2)
            data3, url3, last_url3 = getTownData(suffix2)
            for l in range(len(data3)):
                add3 = data3[l]
                print(add3)
                # suffix3 = url3[l]
                # next3 = []
                # suffix3 = last_url3.replace(last_url3.split('/')[-1], '') + suffix3
                # print(suffix3)
                # data4 = getVillageData(suffix3)
                # for h in range(len(data4)):
                #     next3.append({'name': h})
                next2.append({'name': add3})
            next1.append({'name': add2, 'next': next2})
        next0.append({'name': add1, 'next': next1})
    address.append({'name': add0, 'next': next0})
print(address)
with open("address.txt", "w") as fp:
    fp.write(json.dumps(address, indent=4, ensure_ascii=False))